{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Combiner\n",
    "\n",
    "Combines two models, an SKLearn model and a Tensorflow model for MNIST. The combination does a simple average of the two models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mlogging\u001b[39;49;00m\r\n",
      "logger = logging.getLogger(\u001b[31m__name__\u001b[39;49;00m)\r\n",
      "\r\n",
      "\u001b[34mclass\u001b[39;49;00m \u001b[04m\u001b[32mMnistCombiner\u001b[39;49;00m(\u001b[36mobject\u001b[39;49;00m):\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32m__init__\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, metrics_ok=\u001b[36mTrue\u001b[39;49;00m):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mMNIST Combiner Init called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32maggregate\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, Xs, features_names):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mMNIST Combiner aggregate called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "        logger.info(Xs)\r\n",
      "        \u001b[34mreturn\u001b[39;49;00m (Xs[\u001b[34m0\u001b[39;49;00m]+Xs[\u001b[34m1\u001b[39;49;00m])/\u001b[34m2.0\u001b[39;49;00m\r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize MnistCombiner.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Minikube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!minikube start --memory 4096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context \"minikube\" modified.\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Combiner image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:0.13 seldonio/mnistcombiner_rest:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install Helm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "serviceaccount/tiller created\n",
      "clusterrolebinding.rbac.authorization.k8s.io/tiller created\n",
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!kubectl -n kube-system create sa tiller\n",
    "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
    "!helm init --service-account tiller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install Seldon Core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "namespace/seldon created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Thu May  9 15:26:10 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ClusterRole\n",
      "NAME                          AGE\n",
      "seldon-operator-manager-role  0s\n",
      "\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.106.130.37  <none>       443/TCP  0s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                DESIRED  CURRENT  AGE\n",
      "seldon-operator-controller-manager  1        1        0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                  READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     0s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true   --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for 1 pods to be ready...\n",
      "partitioned roll out complete: 1 new pods have been updated...\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Thu May  9 15:26:40 2019\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP      EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador-admins  ClusterIP     10.98.202.100   <none>       8877/TCP                    0s\n",
      "ambassador         LoadBalancer  10.106.119.175  <pending>    80:30104/TCP,443:31687/TCP  0s\n",
      "\n",
      "==> v1/Deployment\n",
      "NAME        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  3        3        3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-5b89d44544-4l5dx  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-6k9rc  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-zxkh5  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace seldon ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To send requests to Ambassador ingress in another terminal run:\n",
    "    \n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8002:8080\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/clive/work/seldon-core/fork-seldon-core/examples/combiners/mnist_combiner/utils.py:57: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import utils\n",
    "from visualizer import get_graph\n",
    "mnist = utils.download_mnist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"263pt\" height=\"243pt\"\n",
       " viewBox=\"0.00 0.00 263.00 243.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 239)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-239 259,-239 259,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-227 247,-227 247,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"127.5\" y=\"-211.8\" font-family=\"Times,serif\" font-size=\"14.00\">predictor</text>\n",
       "</g>\n",
       "<!-- combiner0 -->\n",
       "<g id=\"node1\" class=\"node\"><title>combiner0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"164,-196 96,-196 96,-160 164,-160 164,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"130\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">combiner</text>\n",
       "</g>\n",
       "<!-- combiner0endpoint -->\n",
       "<g id=\"node2\" class=\"node\"><title>combiner0endpoint</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"48\" cy=\"-106\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">REST</text>\n",
       "</g>\n",
       "<!-- combiner0&#45;&gt;combiner0endpoint -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>combiner0&#45;&gt;combiner0endpoint</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M109.73,-159.697C98.5554,-150.157 84.5922,-138.237 72.7738,-128.148\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"74.8732,-125.339 64.9951,-121.508 70.3284,-130.663 74.8732,-125.339\"/>\n",
       "</g>\n",
       "<!-- skmodel -->\n",
       "<g id=\"node3\" class=\"node\"><title>skmodel</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"161.5,-124 98.5,-124 98.5,-88 161.5,-88 161.5,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"130\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">skmodel</text>\n",
       "</g>\n",
       "<!-- combiner0&#45;&gt;skmodel -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>combiner0&#45;&gt;skmodel</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M130,-159.697C130,-151.983 130,-142.712 130,-134.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"133.5,-134.104 130,-124.104 126.5,-134.104 133.5,-134.104\"/>\n",
       "</g>\n",
       "<!-- tfmodel -->\n",
       "<g id=\"node5\" class=\"node\"><title>tfmodel</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"238.5,-124 179.5,-124 179.5,-88 238.5,-88 238.5,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"209\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">tfmodel</text>\n",
       "</g>\n",
       "<!-- combiner0&#45;&gt;tfmodel -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>combiner0&#45;&gt;tfmodel</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M149.528,-159.697C159.382,-150.965 171.486,-140.24 182.196,-130.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"184.533,-133.356 189.697,-124.104 179.891,-128.117 184.533,-133.356\"/>\n",
       "</g>\n",
       "<!-- skmodelendpoint -->\n",
       "<g id=\"node4\" class=\"node\"><title>skmodelendpoint</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"124\" cy=\"-34\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"124\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">REST</text>\n",
       "</g>\n",
       "<!-- skmodel&#45;&gt;skmodelendpoint -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>skmodel&#45;&gt;skmodelendpoint</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M128.517,-87.6966C127.856,-79.9827 127.061,-70.7125 126.324,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"129.807,-61.7689 125.466,-52.1043 122.833,-62.3667 129.807,-61.7689\"/>\n",
       "</g>\n",
       "<!-- tfmodelendpoint -->\n",
       "<g id=\"node6\" class=\"node\"><title>tfmodelendpoint</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"207\" cy=\"-34\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"207\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">REST</text>\n",
       "</g>\n",
       "<!-- tfmodel&#45;&gt;tfmodelendpoint -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>tfmodel&#45;&gt;tfmodelendpoint</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M208.506,-87.6966C208.285,-79.9827 208.02,-70.7125 207.775,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"211.273,-62.0003 207.489,-52.1043 204.276,-62.2002 211.273,-62.0003\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fb876239240>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_graph(\"mnist_combiner.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "    \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1alpha2\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnistcombo\"\u001b[39;49;00m\r\n",
      "    },\r\n",
      "    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnistcombo\"\u001b[39;49;00m,\r\n",
      "        \u001b[34;01m\"oauth_key\"\u001b[39;49;00m: \u001b[33m\"oauth-key\"\u001b[39;49;00m,\r\n",
      "        \u001b[34;01m\"oauth_secret\"\u001b[39;49;00m: \u001b[33m\"oauth-secret\"\u001b[39;49;00m,\r\n",
      "        \u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n",
      "            {\r\n",
      "                \u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [{\r\n",
      "                    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "                        \u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n",
      "                            {\r\n",
      "                                \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/sk-mnist:0.1\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"skmodel\"\u001b[39;49;00m\r\n",
      "                            },\r\n",
      "                            {\r\n",
      "                                \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/deep-mnist:0.1\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfmodel\"\u001b[39;49;00m\r\n",
      "                            },\r\n",
      "                            {\r\n",
      "                                \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mnistcombiner_rest:0.1\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"combiner\"\u001b[39;49;00m\r\n",
      "                            }\r\n",
      "                        ],\r\n",
      "                        \u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "                    }\r\n",
      "                }],\r\n",
      "                \u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n",
      "                    \u001b[34;01m\"children\"\u001b[39;49;00m: [\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"children\"\u001b[39;49;00m: [],\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"skmodel\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"endpoint\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\u001b[34;01m\"type\"\u001b[39;49;00m : \u001b[33m\"REST\"\u001b[39;49;00m\r\n",
      "\t\t\t    },\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"children\"\u001b[39;49;00m: [],\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfmodel\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"endpoint\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\u001b[34;01m\"type\"\u001b[39;49;00m : \u001b[33m\"REST\"\u001b[39;49;00m\r\n",
      "\t\t\t    },\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m\r\n",
      "\t\t\t}\r\n",
      "\t\t    ],\r\n",
      "                    \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"combiner\"\u001b[39;49;00m,\r\n",
      "                    \u001b[34;01m\"endpoint\"\u001b[39;49;00m: {\r\n",
      "                        \u001b[34;01m\"type\"\u001b[39;49;00m : \u001b[33m\"REST\"\u001b[39;49;00m\r\n",
      "                    },\r\n",
      "\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"COMBINER\"\u001b[39;49;00m\r\n",
      "                },\r\n",
      "                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnistcombo\"\u001b[39;49;00m,\r\n",
      "                \u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "            }\r\n",
      "        ]\r\n",
      "    }\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize mnist_combiner.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/mnistcombo created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f mnist_combiner.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"mnistcombo-mnistcombo-aacd66a\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"mnistcombo-mnistcombo-aacd66a\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/mnistcombo-mnistcombo-aacd66a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADtVJREFUeJzt3X+sVPWZx/HPo0X/4EcUuVAiuLdWgkvQpTAhRM3qRiF2aYKNFopJQ5MN18SaLKZ/KBgDiShmsVQT1yooKRqwNrau/oErQjbaJqY6EAGRXavmilduuFfBlGJCVZ794x6aq975ztyZM3Pm8rxfCZmZ85wfT0Y/98zMd+Z8zd0FIJ6zim4AQDEIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoL7VyoNNmDDBOzs7W3lIIJTu7m59/PHHVsu6DYXfzK6X9JCksyU97u73p9bv7OxUuVxu5JAAEkqlUs3r1v2y38zOlvSfkr4vaYakpWY2o979AWitRt7zz5X0rru/7+5/k/QbSYvyaQtAszUS/gslfTjocU+27CvMrMvMymZW7u/vb+BwAPLUSPiH+lDhG78PdveN7l5y91JHR0cDhwOQp0bC3yNp6qDHUyQdbqwdAK3SSPjfkDTNzL5jZudI+rGkF/JpC0Cz1T3U5+5fmNltkl7SwFDfZnc/kFtnAJqqoXF+d98uaXtOvQBoIb7eCwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFANzdJrZt2Sjkv6UtIX7l7Koym0zsmTJ5P1DRs2JOurVq1K1s1s2D3V6pJLLknWd+zYUbHW2dmZczcjT0Phz/yLu3+cw34AtBAv+4GgGg2/S9phZrvNrCuPhgC0RqMv+69098NmNlHSy2b2v+7+6uAVsj8KXZJ00UUXNXg4AHlp6Mzv7oez2z5Jz0maO8Q6G9295O6ljo6ORg4HIEd1h9/MRpvZ2NP3JS2Q9FZejQForkZe9k+S9Fw2lPMtSdvc/b9z6QpA09Udfnd/X9I/5dgLmmDnzp3J+vr16xvavto4fjPH+d97771k/bLLLqtYW7FiRXLbe+65p66eRhKG+oCgCD8QFOEHgiL8QFCEHwiK8ANB5fGrPlRx6NChZH3JkiXJ+kcffVT3sY8dO5asf/bZZ3XvW0oPp0nSnDlzKtbuuOOO5LYffvhhsr5t27ZkPfWT3mpDnO6erK9duzZZHwk48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzz56DaOP4NN9yQrO/duzfPdoZlzJgxyfrrr7+erI8fPz5Zb+TqTdOnT0/Wr7vuumR9wYIFFWu7du1KbvvAAw8k6zNmzEjWb7755mS9HXDmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOevUeq35YsWLUpuu2/fvrzb+YrLL7+8Yq3aNNYrV65M1quNtbez1PUC9u/fn9y2r68vWf/000/r6qmdcOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaCqjvOb2WZJP5DU5+4zs2XjJT0jqVNSt6TF7p6+QPwI98gjj1SsNXscf/78+cn6U089VbHWyO/pR7prr722Ym3x4sXJbR9++OFkvaenp66e2kktZ/5fS7r+a8vulLTL3adJ2pU9BjCCVA2/u78q6ejXFi+StCW7v0VS+lI1ANpOve/5J7l7ryRltxPzawlAKzT9Az8z6zKzspmV+/v7m304ADWqN/xHzGyyJGW3FX8F4e4b3b3k7qXIHz4B7abe8L8gaVl2f5mk5/NpB0CrVA2/mT0t6TVJ082sx8z+TdL9kuab2Z8lzc8eAxhBrNo85HkqlUpeLpdbdrw8nXfeeRVrx48fb2jf1a6d/9JLLyXr8+bNa+j4ER08eDBZnzlzZrI+bty4ZP3YsWK+9lIqlVQul62WdfmGHxAU4QeCIvxAUIQfCIrwA0ERfiAoLt3dBqpNB81QXvupNkRebfh37NixebZTF878QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4/yZkydPNm3fo0aNStbnzJnTtGOjOU6cOJGsP/7448n67bffnmc7deHMDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBMc6f2bBhQ7LeyOW5Fy5cmKzPnj277n2jPjt27Gho+xtvvDFZb4dx/Go48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFXH+c1ss6QfSOpz95nZsjWSlkvqz1Zb5e7bm9VkO2hkKvP77rsvx05Qq8OHD1esVfu9fbX/3ldddVVdPbWTWs78v5Z0/RDLf+nus7J/Z3TwgTNR1fC7+6uSjragFwAt1Mh7/tvMbJ+ZbTaz83PrCEBL1Bv+X0n6rqRZknol/aLSimbWZWZlMyv39/dXWg1Ai9UVfnc/4u5fuvspSZskzU2su9HdS+5e6ujoqLdPADmrK/xmNnnQwx9KeiufdgC0Si1DfU9LukbSBDPrkbRa0jVmNkuSS+qWdEsTewTQBFXD7+5Lh1j8RBN6aWtmVnQLGKaurq6Ktbfffju57ZQpU+re90jBN/yAoAg/EBThB4Ii/EBQhB8IivADQXHpboxYvb29yfoHH3xQsTZt2rTktq+88kqyfs455yTrIwFnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+TLWfaK5bt65i7cSJE8ltDxw4kKxPnz49WY8qdeltSVq8eHGynvrZ7mOPPZbcdtKkScn6mYAzPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTh/5oILLkjWzzqr/r+Tt9ySntbg0ksvTdZnzJhR97FHsm3btiXrr732Wt37ZvYozvxAWIQfCIrwA0ERfiAowg8ERfiBoAg/EFTVcX4zmyrpSUnflnRK0kZ3f8jMxkt6RlKnpG5Ji939WPNaLdbKlSsr1tasWZPc9ujRo8n62rVrk/VHH300WR83blyyXqTdu3dXrN17773JbXfu3NnQsdevX1+xdsUVVzS07zNBLWf+LyT93N3/UdI8ST8zsxmS7pS0y92nSdqVPQYwQlQNv7v3uvue7P5xSQclXShpkaQt2WpbJN3QrCYB5G9Y7/nNrFPS9yT9SdIkd++VBv5ASJqYd3MAmqfm8JvZGEm/k7TC3f8yjO26zKxsZuX+/v56egTQBDWF38xGaSD4W93999niI2Y2OatPltQ31LbuvtHdS+5e4scUQPuoGn4zM0lPSDro7hsGlV6QtCy7v0zS8/m3B6BZzN3TK5hdJekPkvZrYKhPklZp4H3/byVdJOmQpB+5e3JMq1QqeblcbrTntjN79uxkfe/evQ3tf8mSJcn63XffXbE2cWL6o5i+viFfsNXsrrvuSta3b99esfb5558nt602hHn11Vcn65s2bapYO1NfhZZKJZXLZatl3arj/O7+R0mVdnbtcBoD0D74hh8QFOEHgiL8QFCEHwiK8ANBEX4gqKrj/Hk6U8f5n3322WT91ltvTdY/+eSTPNv5imrfQdizZ09D+6/heyIVa5MnT05u++CDDybrN910U7Ie0XDG+TnzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQTNGdg2rjzdXG2hcuXJisv/POO8Pu6bRGx/GrOffcc5P1devWVawtX748ue3o0aPr6gm14cwPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzt8CF198cbL+4osvJuvPPPNMsr5169aKtQMHDiS3rTZOv3r16mR93rx5yXq1a+ujOJz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoqtftN7Opkp6U9G1JpyRtdPeHzGyNpOWS+rNVV7l75cnYdeZetx9oF8O5bn8tX/L5QtLP3X2PmY2VtNvMXs5qv3T3B+ptFEBxqobf3Xsl9Wb3j5vZQUkXNrsxAM01rPf8ZtYp6XuS/pQtus3M9pnZZjM7v8I2XWZWNrNyf3//UKsAKEDN4TezMZJ+J2mFu/9F0q8kfVfSLA28MvjFUNu5+0Z3L7l7qaOjI4eWAeShpvCb2SgNBH+ru/9ektz9iLt/6e6nJG2SNLd5bQLIW9Xw28A0q09IOujuGwYtHzzF6g8lvZV/ewCapZZP+6+U9BNJ+83szWzZKklLzWyWJJfULemWpnQIoClq+bT/j5KGGjdMjukDaG98ww8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBU1Ut353ows35JHwxaNEHSxy1rYHjatbd27Uuit3rl2ds/uHtN18trafi/cXCzsruXCmsgoV17a9e+JHqrV1G98bIfCIrwA0EVHf6NBR8/pV17a9e+JHqrVyG9FfqeH0Bxij7zAyhIIeE3s+vN7P/M7F0zu7OIHioxs24z229mb5pZoVMKZ9Og9ZnZW4OWjTezl83sz9ntkNOkFdTbGjP7KHvu3jSzfy2ot6lm9j9mdtDMDpjZv2fLC33uEn0V8ry1/GW/mZ0t6R1J8yX1SHpD0lJ3f7uljVRgZt2SSu5e+Jiwmf2zpL9KetLdZ2bL/kPSUXe/P/vDeb6739Emva2R9NeiZ27OJpSZPHhmaUk3SPqpCnzuEn0tVgHPWxFn/rmS3nX39939b5J+I2lRAX20PXd/VdLRry1eJGlLdn+LBv7nabkKvbUFd+919z3Z/eOSTs8sXehzl+irEEWE/0JJHw563KP2mvLbJe0ws91m1lV0M0OYlE2bfnr69IkF9/N1VWdubqWvzSzdNs9dPTNe562I8A81+087DTlc6e6zJX1f0s+yl7eoTU0zN7fKEDNLt4V6Z7zOWxHh75E0ddDjKZIOF9DHkNz9cHbbJ+k5td/sw0dOT5Ka3fYV3M/ftdPMzUPNLK02eO7aacbrIsL/hqRpZvYdMztH0o8lvVBAH99gZqOzD2JkZqMlLVD7zT78gqRl2f1lkp4vsJevaJeZmyvNLK2Cn7t2m/G6kC/5ZEMZD0o6W9Jmd7+35U0Mwcwu1sDZXhqYxHRbkb2Z2dOSrtHAr76OSFot6b8k/VbSRZIOSfqRu7f8g7cKvV2jgZeuf5+5+fR77Bb3dpWkP0jaL+lUtniVBt5fF/bcJfpaqgKeN77hBwTFN/yAoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwT1/+IzRh9/UOzWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'meta': {'puid': 'gdqbdu064ae8qnl7cf8g1l9ceb', 'tags': {}, 'routing': {'combiner': -1}, 'requestPath': {'tfmodel': 'seldonio/deep-mnist:0.1', 'skmodel': 'seldonio/sk-mnist:0.1', 'combiner': 'seldonio/mnistcombiner_rest:0.1'}, 'metrics': []}, 'data': {'names': ['t:0', 't:1', 't:2', 't:3', 't:4', 't:5', 't:6', 't:7', 't:8', 't:9'], 'ndarray': [[0.9853938817977905, 3.674550247412256e-11, 1.44667967560963e-06, 1.756110691530921e-06, 3.302479845146422e-09, 0.014028188772499561, 2.3320741604493378e-07, 7.86559226639838e-08, 0.0005743441870436072, 5.9187676981764525e-08]]}}\n"
     ]
    }
   ],
   "source": [
    "utils.predict_rest_mnist(mnist,\"mnistcombo\",\"seldon\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
